{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mnist\n",
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> import lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "# Imports\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "tf.logging.debug('ALL')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-0bae8f1cfcf7>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((55000, 784), (10000, 784))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = 'data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "train_data = mnist.train.images # Returns np.array\n",
    "train_labels = np.asarray(mnist.train.labels, dtype=np.int32)\n",
    "eval_data = mnist.test.images # Returns np.array\n",
    "eval_labels = np.asarray(mnist.test.labels, dtype=np.int32)\n",
    "\n",
    "train_data.shape, eval_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> add regularizer, use l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "regularizer = tf.contrib.layers.l2_regularizer(0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> define input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'StopGradient:0' shape=(?, 10) dtype=float32>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_in_features = train_data.shape[1]\n",
    "num_out_classes = train_labels.shape[1]\n",
    "var_w = np.sqrt(2/(num_in_features + num_out_classes))\n",
    "x = tf.placeholder(dtype = tf.as_dtype(train_data.dtype), shape = (None, num_in_features))\n",
    "y = tf.placeholder(dtype = tf.as_dtype(train_data.dtype), shape = (None, num_out_classes))\n",
    "\n",
    "tf.stop_gradient(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Reshape:0' shape=(?, 28, 28, 1) dtype=float32>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_reshape = tf.reshape(x, (-1, 28, 28, 1))\n",
    "x_reshape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> hiden layer 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_conv1 = tf.get_variable('w_conv1', initializer=tf.truncated_normal((5, 5, 1, 1), mean = 0, stddev = var_w), regularizer = regularizer)\n",
    "b_conv1 = tf.get_variable('b_conv1', initializer=tf.truncated_normal((1, 28, 28, 1), mean = 0, stddev = var_w), regularizer = regularizer)\n",
    "conv2d1 = tf.nn.conv2d(x_reshape, w_conv1, strides = [1 , 1, 1, 1], padding='SAME', data_format='NHWC') + b_conv1\n",
    "active1 = tf.nn.selu(conv2d1)\n",
    "\n",
    "pool1 = tf.nn.max_pool(active1, (1, 2, 2, 1), (1, 2, 2, 1), padding = 'SAME')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> hiden layer 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2 = tf.get_variable('w2', initializer=tf.random_normal((196, 784)), regularizer = regularizer)\n",
    "b2 = tf.get_variable('b2', initializer=tf.random_normal((1, 784)), regularizer = regularizer)\n",
    "logits2 = tf.matmul(tf.reshape(pool1, (-1, 196)), w2) + b2\n",
    "active2 = tf.nn.tanh(logits2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> output layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "w3 = tf.get_variable('w3', initializer=tf.random_normal((784, 10)), regularizer = regularizer)\n",
    "b3 = tf.get_variable('b3', initializer=tf.random_normal((1, 10)), regularizer = regularizer)\n",
    "logits3 = tf.matmul(active2, w3) + b3\n",
    "loss = tf.nn.softmax_cross_entropy_with_logits_v2(logits = logits3, labels = y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> add regularizer to loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_with_reg = loss + tf.losses.get_regularization_loss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> define optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_step = tf.Variable(0, trainable=False)\n",
    "starter_learning_rate = 0.0001\n",
    "learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step = global_step,\n",
    "                                           decay_steps = 100, decay_rate = 0.6, staircase=False)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=0.001)\n",
    "train_op = optimizer.minimize(\n",
    "    loss=loss_with_reg,\n",
    "    global_step=tf.train.get_global_step())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> add accuracy metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(logits3, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> init variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train\n",
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " loss: 16.670944\n",
      " loss: 2.7039723\n",
      " loss: 0.5764593\n",
      " loss: 0.15450159\n",
      " loss: 0.072345406\n",
      " loss: 0.058572076\n",
      " loss: 0.05656635\n",
      " loss: 0.05501545\n",
      "21.8 s ± 71.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "loss_list = []\n",
    "for i in range(2000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(1024)\n",
    "    _, cur_loss = sess.run((train_op, loss_with_reg), feed_dict={x: batch_xs, y: batch_ys})\n",
    "    loss_list.append(cur_loss)\n",
    "    if (i % 100) == 0:\n",
    "        print('\\r', 'loss:', cur_loss[-1], end = '')\n",
    "    \n",
    "print() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "predict\n",
    "-------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train [array([0.14324224, 0.05446461, 0.05456075, ..., 0.05399986, 0.05985837,\n",
      "       0.30474997], dtype=float32), 0.9976182]\n",
      "test  [array([0.05403848, 0.05422154, 0.05428267, ..., 0.05417876, 0.05439575,\n",
      "       0.05391547], dtype=float32), 0.9844]\n"
     ]
    }
   ],
   "source": [
    "  # Test trained model\n",
    "# correct_prediction = tf.equal(tf.argmax(logits2, 1), tf.argmax(y, 1))\n",
    "# accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print('train', sess.run([loss_with_reg, accuracy], feed_dict={x: mnist.train.images,\n",
    "                                      y: mnist.train.labels}))\n",
    "print('test ', sess.run([loss_with_reg, accuracy], feed_dict={x: mnist.test.images,\n",
    "                                      y: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
